import asyncio
import aiohttp
import json
import logging
from motor.motor_asyncio import AsyncIOMotorClient
# 日志格式化
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
# 列表页URL
INDEX_URL = 'https://spa5.scrape.center/api/book/?limit={limit}&offset={offset}'
# 详情页URL
DETAIL_URL = 'https://spa5.scrape.center/api/book/{id}/'
# 每页展示数量
LIMIT = 18
# 爬取的最大页码数
PAGE = 100
# 并发量
CONCURRENCY = 20
# 声明并发量信号，用于控制最大并发数量
semaphore = asyncio.Semaphore(CONCURRENCY)
session = None

# 封装通用的爬取方法
async def scrape_api(url):
    async with semaphore:
        try:
            logging.info('scraping %s', url)
            async with session.get(url) as response:
                return await response.json()
        except aiohttp.ClientError:
            logging.error('error occurred while scraping %s', url)

# 封装爬取列表页
async def scrape_index(page):
    url = INDEX_URL.format(limit=LIMIT, offset=(page - 1) * LIMIT)
    return await scrape_api(url)

# 配置MongoDB数据库信息
MONGO_CONNECTION_STRING = 'mongodb://localhost:27017/'
MONGO_DB_NAME = 'books_spa5'
MONGO_COLLECTION_NAME = 'spa5'
# 连接MongoDB数据库
client = AsyncIOMotorClient(MONGO_CONNECTION_STRING)
db = client[MONGO_DB_NAME]
collection = db[MONGO_COLLECTION_NAME]

# 封装保存数据函数
async def save_data(data):
    logging.info('saving data %s', data)
    if data:
        # 这是MongoDB数据库的写法
        return await collection.update_one({
            'id': data.get('id')
        }, {
            '$set': data
        }, upsert=True)

# 封装爬取详情页，并实现将数据保存到数据库
async def scrape_detail(id):
    url = DETAIL_URL.format(id=id)
    data = await scrape_api(url)
    await save_data(data)

# 封装总执行方法
async def main():
    global session
    session = aiohttp.ClientSession()
    # 将爬取列表页的任务声明为tasks列表
    scrape_index_tasks = [asyncio.ensure_future(scrape_index(page)) for page in range(1, PAGE + 1)]
    # 使用异步集中执行tasks列表任务
    results = await asyncio.gather(*scrape_index_tasks)
    # 提取列表页返回数据中的id，这个id就是拼接详情页url使用的
    ids = []
    for index_data in results:
        if not index_data: continue
        for item in index_data.get('results'):
            ids.append(item.get('id'))
    logging.info('results %s', json.dumps(results, ensure_ascii=False, indent=2))
    # 将爬取详情页的任务声明为tasks列表
    scrape_detail_tasks = [asyncio.ensure_future(scrape_detail(id)) for id in ids]
    # 使用异步集中执行tasks列表任务
    await asyncio.gather(*scrape_detail_tasks)
    # 关闭会话连接
    await session.close()

if __name__ == '__main__':
    asyncio.run(main())